/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */

package MetricsRetriever;

import General.Globals;
import General.RequestMaker;
import General.Utilities;
import SVNSupport.SVNManager;
import java.io.File;
import java.text.Normalizer;
import java.text.Normalizer.Form;
import java.util.ArrayList;

/**
 *
 * @author Manuel
 */
public abstract class Retriever extends Thread {

    protected Project currentProject;
    protected ProjectPackage codePackage;
    protected String currentPage;
    protected String currentLink;
    protected DataManager dmgr;
    protected String forgeName;
    protected String codeLink;
    protected ArrayList<File> classFiles;
    protected ArrayList<File> javaFiles;
    protected FileManager fmgr;
    protected SVNManager svnmgr;


    public Retriever(DataManager dm){
        dmgr = dm;
        javaFiles = new ArrayList<File>();
        classFiles = new ArrayList<File>();
        svnmgr = new SVNManager();
    }

    public abstract void retrieveContextData();

    public abstract ProjectPackage selectArchieveLink();

    public abstract String selectRepositoryLink();
    //checks where the project is located and invokes the according retrieval method
    public void run(){

        while(dmgr.getLinksLeft(this.forgeName)>0){

            //process and setup a new project
            currentProject = new Project();
            fmgr = new FileManager(currentProject);
            currentLink = dmgr.processNextLink(this.forgeName);
            if(currentLink == null){
                break;
            }

            //get the project homepage and retrieve contextual data and continue only if enough data is provided
            currentPage = RequestMaker.getPage(currentLink);
            this.retrieveContextData();
            Utilities.cleanMemory();
            if(filterProject(Globals.CHECK_CONTEXT)){

                //choose the right archieve to download the source code, continue only if some classes are downloaded
                this.retrieveCodeData();
                Utilities.cleanMemory();

                //check whether the files have been downloaded correctly
                if(this.filterProject(Globals.CHECK_CODE)){
                    boolean checkedOut = this.retrieveRepositoryData();
                    Utilities.cleanMemory();
                    //retrieve SVN repository data
                    if(this.filterProject(Globals.CHECK_REPO) && checkedOut){

                        //retrieve metricsfrom the class and java files
                        Globals.counter++;
                        RetrievalManager.noSuccess++;
                        //WRITE TO DB
                        System.out.println("COMPLETED "+currentProject.getName()+"");
                        Utilities.writeToFile("\n\n"+Globals.counter+" "+currentProject.getName(), Globals.projectFile, true);
                        Utilities.cleanMemory();

                    }


                }
            }
            Utilities.cleanMemory();
        
        }
        
    }

    public boolean retrieveRepositoryData(){
        boolean success = false;
        boolean checkedOut = false;
        boolean logged = false;
        if(this.selectRepositoryLink() == null || this.selectRepositoryLink().isEmpty())
            return false;
        //if it is svn
        else if(this.selectRepositoryLink().startsWith("svn")){
            String link = this.selectRepositoryLink().substring(3);

            checkedOut = svnmgr.checkout(link, currentProject.getSourceDirectory());
            ArrayList log = svnmgr.getHistory(link, -1);
            if(log != null){
                currentProject.setRepositoryHistory(log);
                currentProject.setRepositoryType(Globals.SVN);
                logged = true;
            }
        }
        if(checkedOut && logged){
            currentProject.setSources(fmgr.process(currentProject.getSourceDirectory()));
            success = true;
        }
        return success;
        
    }
    public void retrieveCodeData() {
        codePackage = this.selectArchieveLink();
        if(codePackage != null){
            this.currentProject.setFirstCodePackage(codePackage);
            codeLink = codePackage.getName();
            if(codeLink == null || codeLink.isEmpty())
                return;

            //download the chosen file
            String codeFile = RequestMaker.storeFile(codeLink, Globals.DATA_DIRECTORY);


            //extract the chosen file and store the classfiles
            currentProject.setClasses(fmgr.process(codeFile));
        }
    }


    public boolean filterProject(int type){
        //TODO CHANGE AS SYSTEM SUPPORTS OTHER LANGUAGES
        if(type == Globals.CHECK_CONTEXT){
            if(currentProject.isOfLanguage("java")){
                return true;
            }
            RetrievalManager.noFailedContext++;
            return false;
        }

        if(type == Globals.CHECK_CODE){
            if(currentProject.getClassFiles() != null && currentProject.getClassFiles().size()>10){
                return true;
            }
            RetrievalManager.noFailedByte++;
            System.out.println("BYTECODE RETRIEVAL FAILED FOR "+currentProject.getName());
            fmgr.cleanUp();
            return false;
        }

        if(type == Globals.CHECK_REPO){
            if(currentProject.getSourceFiles() != null && currentProject.getSourceFiles().size()>10){
                return true;
            }
            RetrievalManager.noFailedCode++;
            System.out.println("SOURCECODE RETRIEVAL FAILED FOR "+currentProject.getName());
            fmgr.cleanUp();
            return false;
        }
        return true;
    }

     public String parseCharacterData(String startString, String endString, String page,boolean filter,boolean excludeTag) {
        String characters = "";
        int startIndex = page.indexOf(startString);
        if(excludeTag)
            startIndex = page.indexOf(startString)+startString.length();
        int endIndex = page.indexOf(endString,startIndex);
        if(startIndex >= 0 && endIndex >= 0)
            characters = page.substring(startIndex,endIndex);
            if(filter)
                characters = filterHtml(characters);
        return characters;
    }

    public String filterHtml(String htmlText){

        //remove all javascript
        htmlText = htmlText.replaceAll("\\<script.*?\\>.+\\</script>", "");

        //remove all tags
        htmlText = htmlText.replaceAll("\\<.*?\\>", "");

        //remove all other markers
        htmlText = htmlText.replaceAll("&.+;","");

        htmlText = htmlText.toLowerCase();

        //normalize according to Unicode Standard
        htmlText = Normalizer.normalize(htmlText, Form.NFD );

        return htmlText;
    }

   


}
